#coding:utf-8
'''
* coder  : dzlua
* email  : 505544956@qq.com
* module : spider
* path   : jbdq
* file   : jbdq.py
* time   : 2017-11-13 13:59:17
'''
#--------------------#
import sys, codecs, os, time
from bs4 import BeautifulSoup
#----------#
sys.path.append("..")
#----------#
from spider import Spider
from database import DataBase
#--------------------#

#--------------------#
class Jbdq(Spider, DataBase):
    def __init__(self, host, user, pwd, db, threads_work = 4, tasks = 0):
        DataBase.__init__(self, host, user, pwd, db)
        Spider.__init__(self, threads_work, tasks)

        sql = "CREATE TABLE IF NOT EXISTS " + self.tb + \
            " (id INT AUTO_INCREMENT PRIMARY KEY," \
            " type1 TEXT," \
            " type2 TEXT," \
            " type3 TEXT," \
            " type4 TEXT," \
            " href TEXT," \
            " info TEXT )" \
            " DEFAULT CHARSET=utf8;"
        DataBase.execute(self, sql)
        #

        self.host = 'http://www.zhzyw.org/'
    def path(self, path):
        return os.path.join(self.host, path)
    def start(self):
        self.put(
            self.__on_page1,
            self.path('jbdq.html') )
    def __on_page1(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find(id="main")
        if not div:
            self.log('__on_page1, no #main. %s' % url)
            return
        #
        h3 = None
        for ele in div.children:
            if ele.name == 'h3':
                a = ele.find('a')
                h3 = a.text
            elif ele.name == 'a':
                info = {
                    'type1': h3,
                    'type2': ele.text,
                }
                href = self.path(ele.get('href')[1:])
                self.put(self.__on_page2, href, None, info)
    def __on_page2(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find('div', class_ = ['ullist'])
        if not div:
            info = {
                'type1': args['type1'],
                'type2': args['type2'],
                'type3': '',
            }
            self.__on_page3(ret, url, info)
            return
        #
        ul = div.find('ul')
        if not ul:
            self.log('__on_page2, no ul %s' % url)
            return
        #
        for ele in ul.find_all('li'):
            if not ele:
                continue
            #
            a = ele.find('a')
            if not a:
                continue
            #
            info = {
                'type1': args['type1'],
                'type2': args['type2'],
                'type3': a.text,
            }
            href = self.path(a.get('href')[1:])
            self.put(self.__on_page3, href, None, info)
    def __on_page3(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find('div', class_ = ['ullist01'])
        if not div:
            self.log('__on_page3, no div ullist01 %s' % url)
            return
        #
        ul = div.find('ul')
        if not ul:
            self.log('__on_page3, no ul %s' % url)
            return
        #
        for ele in ul.find_all('li'):
            if not ele:
                continue
            #
            a = ele.find('a')
            if not a:
                continue
            #
            href = self.path(a.get('href')[1:])
            info = {
                'type1': args['type1'],
                'type2': args['type2'],
                'type3': args['type3'],
                'type4': a.text,
            }
            self.put(self.__on_page4, href, None, info)
    def __on_page4(self, ret, url, args):
        if not ret:
            return
        soup = BeautifulSoup(ret, "lxml")
        #
        div = soup.find('div', class_ = ['webnr'])
        if not div:
            self.log('__on_page4, no div webnr %s' % url)
            return
        #
        text = ''
        for string in div.strings:
            text += (string + '\n')
        #
        info = {
            'type1': args['type1'],
            'type2': args['type2'],
            'type3': args['type3'],
            'type4': args['type4'],
            'info': text,
            'href': url,
        }

        self.save(self.__on_save, None, info)

    def __on_save(self, conn, data, args):
        print args['type1'],args['type2'],args['type3'],args['type4'],args['href']
        
        sql = "INSERT INTO " + self.tb + \
            " (type1, type2, type3, type4, href, info)" \
            " VALUES(%s,%s,%s,%s,%s,%s);"
        param = ( args['type1'],args['type2'],args['type3'],
            args['type4'],args['href'],args['info'] )
        try:
            cursor = conn.cursor()
            cursor.execute(sql, param)
            cursor.close()
            conn.commit()
        except conn.Error, e:
            self.log('__on_save error. %d %s, %s ' % (e[0], e[1], data))
    #--------------------#
    def write_to_file(self, path):
        sql = 'SELECT * FROM %s;' % (self.tb)
        for r in self.select(sql):
            ff = os.path.join(path, r[1])
            ff = os.path.join(ff, r[2])
            ff = os.path.join(ff, r[3])
            ff = os.path.join(ff, r[4])
            ff += '.txt'

            bok = self.create_file(ff, r[6])
            self.log('%s %s %s' % (r[0], bok, ff), not bok)
    def create_file(self, file, content=u''):
        if os.path.exists(file):
            return False
        path = os.path.dirname(file)
        self.mkdirs(path)
        #
        fp = codecs.open(file, 'w', 'utf-8')
        fp.write(content)
        fp.close()

        return True
    def mkdirs(self, path):
        try:
            os.makedirs(path)
        except:
            pass
#--------------------#
